package me.crawler;



import java.util.*;
import java.util.regex.Pattern;
import java.net.*;
import java.io.*;

public class Crawler implements Runnable  {
	
	
	//Constants for robot.txt
    public static final String SEARCH = "Search";
    public static final String STOP = "Stop";
    public static final String DISALLOW = "Disallow:";
    
    
    public static final int    SEARCH_LIMIT = 100000;
    
    private static int errorCount = 0;
    
    
    ///"http://en.wikipedia.org/wiki";
    private  String  delimiter = null;
   

	private static Vector  <String> linkQueue;
    // URLs already searched
	private static Vector <String> linksSearched;
    // URLs which match
	private static Vector <String> vectorMatches;
    
    public Crawler(){
    // URLs to be searched



	// initialize search data structures
	linkQueue = new Vector <String>();
	linksSearched = new Vector <String>();
	vectorMatches = new Vector <String>();

	// set default for URL access
	URLConnection.setDefaultAllowUserInteraction(false);
	
    }

   
    
    public  String getDelimiter() {
		return delimiter;
	}


	public  void setDelimiter(String delimiter) {
		this.delimiter = delimiter;
		
		if(linkQueue.isEmpty())
			linkQueue.addElement(delimiter);
	}
    
    public static int getErrorCount(){
    	return errorCount;
    }
	
    
    Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpeg"
            + "|png|tiff?|mid|mp2|mp3|mp4"
            + "|wav|avi|mov|mpeg|ram|m4v|pdf"
            + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

    
    
    
    public boolean validUrl(String url){
		
        String link = url.toLowerCase();
        if (filters.matcher(link).matches()) 
                return false;
        
        if (link.startsWith("wiki"))
            return true;
        
        if(link.length()>3){
        	String dot = link.substring(3, 4);
        if(dot.contains("."))
        	return false;
        }
        
        if (!link.contains("?")) 
            return true;
        
        return false;
    }


    public void run() {
    	
	    //searchThread = new Thread(this);	
		URL url= null;
		try {
			url = new URL(delimiter);
		} catch (MalformedURLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		String strTargetType = "text/html";
		int numberSearched = 0;
		int numberFound = 0;
	
		if (url.toString().length() == 0) {
			System.out.println("ERROR: must enter a starting URL");
		    return;
		}
	
		// initialize search data structures
		System.out.println("Adding : " +url.toString());
		
	
		while ((linkQueue.size() > 0) ) {
		    // get the first element from the to be searched list
		    try {
		    	if (!linkQueue.isEmpty()){
		    		url = new URL( linkQueue.elementAt(0));
				    linkQueue.removeElementAt(0);
				    
				    if(!linksSearched.contains(url.toString()))
				    		linksSearched.addElement(url.toString());
				    else 
				    	break;		    	
		    	}
			} catch (MalformedURLException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			}
	
		    System.out.println("searching " + url);
	
		   
		    	
	
		
	
		    // mark the URL as searched (we want this one way or the other)

	
		    // can only search http: protocol URLs
		    if (url.getProtocol().compareTo("http") != 0) 
			break;
	
		    // test to make sure it is before searching
	//	    if (!robotSafe(url))
	//		break;
	
		    try {
			// try opening the URL
			URLConnection urlConnection = url.openConnection();
	
			//urlConnection.setAllowUserInteraction(false);
	
			InputStream urlStream = url.openStream();
	//		String type 
	//		  = URLConnection.guessContentTypeFromStream(urlStream);
	//		if (type == null)
	//		    break;
	//		if (type.compareTo("text/html") != 0) 
	//		    break;
	
			// search the input stream for links
			// first, read in the entire URL
			byte b[] = new byte[1000];
			int numRead = urlStream.read(b);
			String content = new String(b, 0, numRead);
				
			while (numRead != -1) {
	
			    numRead = urlStream.read(b);
			    if (numRead != -1) {
				String newContent = new String(b, 0, numRead);
				content += newContent;
			    }
			}
			urlStream.close();
	
			String lowerCaseContent = content.toLowerCase();
	
			int index = 0;
			while ((index = lowerCaseContent.indexOf("<a", index)) != -1)
			{
			    if ((index = lowerCaseContent.indexOf("href", index)) == -1) 
				break;
			    if ((index = lowerCaseContent.indexOf("=", index)) == -1) 
				break;
			    if ((index = lowerCaseContent.indexOf("/", index)) == -1) 
				break;
	//		    if ((index = lowerCaseContent.indexOf(":", index)) == -1) 
	//				break;
	////		    
			    
	
	
			    index++;
			    String remaining = content.substring(index);
			    
			    
	
			    StringTokenizer st 
			      = new StringTokenizer(remaining, "\'\t\n\r\">#");
			    String strLink = st.nextToken();
			    String link = null;
			    
			    URL urlLink;
			    try {
			    	
			    	if (strLink.startsWith("http"))
			    		urlLink = new URL(strLink);	    		
			    	else
			    		urlLink = new URL(new URL(delimiter), strLink);
				
			    	
			    	 link = urlLink.toString();
				//System.out.println(strLink);
			    } catch (MalformedURLException e) {
			    	System.err.println("ERROR: bad URL " + link);
			    	errorCount++;
			    	//even though we get an error accessing the link we want to continue searching rest of the Queue
			    	continue;
			    }
	
			    // only look at http links
	//		    if (urlLink.getProtocol().compareTo("http") != 0)
	//			break;
	
	
			    // if another page, add to the end of search list
	//			if (strType == null)
	//			    break;
	//			if (strType.compareTo("text/html") == 0) {
				    // check to see if this URL has already been 
				    // searched or is going to be searched
				    if ((!linksSearched.contains(link)) && (!linkQueue.contains(link))&& validUrl(strLink)) {
				    		linkQueue.addElement(link);
				    	
				    
				    if (vectorMatches.contains(link) == false) {
					System.out.println("ADDING : "+ link);
					vectorMatches.addElement(link);
					numberFound++;
					if (numberFound >= SEARCH_LIMIT)
					    break;
				    }
				    }
	//			}
			}
		    } catch (IOException e) {
		    	System.out.println("ERROR: couldn't open URL " + url);
			
		    }
	
		    numberSearched++;
		    System.out.println(linkQueue.size() + " Links left to search");
		    if (numberSearched >= SEARCH_LIMIT)
			break;
		}
	
		if (numberSearched >= SEARCH_LIMIT || numberFound >= SEARCH_LIMIT)
			System.out.println("reached search limit of " + SEARCH_LIMIT);
		else
			System.out.println("done");	
	    }
    
    
    boolean robotSafe(URL url) {
    	String strHost = url.getHost();

    	// form URL of the robots.txt file
    	String strRobot = "http://" + strHost + "/robots.txt";
    	URL urlRobot;
    	try { 
    	    urlRobot = new URL(strRobot);
    	} catch (MalformedURLException e) {
    	    // something weird is happening, so don't trust it
    	    return false;
    	}

    	String strCommands;
    	try {
    	    InputStream urlRobotStream = urlRobot.openStream();

    	    // read in entire file
    	    byte b[] = new byte[10000];
    	    int numRead = urlRobotStream.read(b);
    	    strCommands = new String(b, 0, numRead);
    	    while (numRead != -1) {
    		numRead = urlRobotStream.read(b);
    		if (numRead != -1) {
    		    String newCommands = new String(b, 0, numRead);
    		    strCommands += newCommands;
    		}
    	    }
    	    urlRobotStream.close();
    	} catch (IOException e) {
    	    // if there is no robots.txt file, it is OK to search
    	    return true;
    	}

    	// assume that this robots.txt refers to us and 
    	// search for "Disallow:" commands.
    	String strURL = url.getFile();
    	int index = 0;
    	while ((index = strCommands.indexOf(DISALLOW, index)) != -1) {
    	    index += DISALLOW.length();
    	    String strPath = strCommands.substring(index);
    	    StringTokenizer st = new StringTokenizer(strPath);

    	    if (!st.hasMoreTokens())
    		break;
    	    
    	    String strBadPath = st.nextToken();

    	    // if the URL starts with a disallowed path, it is not safe
    	    if (strURL.indexOf(strBadPath) == 0)
    		return false;
    	}
    	return true;
        }
    
    
    
    
    
}